/*
 * FileName：SouhuProcessor.java 
 * <p>
 * Copyright (c) 2017-2020 <a href="https://www.smartwx.info">hermit(1154808491@qq.com)</a>.
 * <p>
 * Licensed under the GNU General Public License, Version 3 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * <p>
 * http://www.gnu.org/licenses/gpl-3.0.html
 * <p>
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * 
 */
package com.smart.webmagic.processor;

import com.smart.common.util.StringUtil;
import com.smart.wx.wxcms.entity.WxcmsNews;
import com.smart.wx.wxcms.service.WxcmsNewsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Component
public class SouhuProcessor implements PageProcessor {

    @Resource
    private WxcmsNewsService wxcmsNewsService;
    // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);
    private Logger logger = LoggerFactory.getLogger(SouhuProcessor.class);

    @Override
    public Site getSite() {
        return site;
    }

    @Override
    public void process(Page page) {
        if("http://news.sohu.com/".equals(page.getUrl().get())){
            List<String> allNews = page.getHtml().xpath("//div[@class='choice-list']/ul/li/a").all();
            WxcmsNews wxcmsNews;
            for (String news : allNews) {
                wxcmsNews = new WxcmsNews();
                wxcmsNews.setNewsOrigin("搜狐");
                wxcmsNews.setTitle(getTitle(news));
                wxcmsNews.setNewsUrl(getUrl(news));
                wxcmsNews.setNewsType("搜狐精选");
                page.addTargetRequest(getUrl(news));
                try {
                    wxcmsNewsService.insert(wxcmsNews);
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                }
            }
        } else {
            WxcmsNews wxcmsNews=new WxcmsNews();
            wxcmsNews.setNewsUrl(page.getUrl().get());
            String ppublishTime=page.getHtml().xpath("//span[@id='news-time']/text()").get();
            if(StringUtil.isBlank(ppublishTime)){
                ppublishTime=page.getHtml().xpath("//span[@class='l time']/text()").get();
            }
            wxcmsNews.setPublishTime(ppublishTime);
            List<Selectable> nodes = page.getHtml().xpath("//div[@class='location area']/a").nodes();
            wxcmsNews.setNewsTypeUrl("http:" + nodes.get(nodes.size()-1).xpath("/a/@href").get());
            wxcmsNews.setNewsType(nodes.get(nodes.size()-1).xpath("/a/text").get());
            try {
                wxcmsNewsService.update(wxcmsNews);
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
            }
        }

    }

    /**
     * 使用正则表达式提取title
     *
     * @param msg
     * @return
     */
    public static String getTitle(String msg) {
        Pattern p = Pattern.compile("title=\".*?\" ");
        Matcher m = p.matcher(msg);
        while (m.find()) {
            String group = m.group();
            return StringUtil.substring(group, 7, group.length() - 9);
        }
        return null;
    }

    /**
     * 使用正则表达式提取url
     *
     * @param msg
     * @return
     */
    public static String getUrl(String msg) {
        Pattern p = Pattern.compile("href=\".*?\" ");
        Matcher m = p.matcher(msg);
        while (m.find()) {
            String group = m.group();
            return "http:" + StringUtil.substring(group, 6, group.length() - 8);
        }
        return null;
    }

    public static void main(String[] args) {
        List<String> urlList = new ArrayList<>();
        urlList.add("http://www.sohu.com/a/289590925_100001551");
        for (String url : urlList) {
            Spider.create(new SouhuProcessor()).addUrl(url).thread(2).run();
        }
//        long startTime, endTime;
//        System.out.println("开始爬取...");
//        startTime = System.currentTimeMillis();
//        Spider.create(new TouTiaoProcessor()).addUrl("http://news.sohu.com/").thread(5).run();
//        endTime = System.currentTimeMillis();
//        System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了"+count+"条记录");
//        String str="var _data = {\"real_time_news\":[{\"open_url\":\"\\/group\\/6583133443222143491\\/\",\"group_id\":\"6583133443222143491\",\"image_url\":\"\\/\\/p3.pstatp.com\\/list\\/240x240\\/pgc-image\\/1532754015451a8bbb0fcde\",\"title\":\"\\u4e0b\\u5468\\u4e00MH370\\u6700\\u7ec8\\u8c03\\u67e5\\u516c\\u5e03\\u524d\\uff0c\\u4e86\\u89e3\\u4e00\\u4e0b\\u7a7a\\u96be\\u8fc7\\u540e\\u6536\\u62fe\\u60b2\\u4f24\\u7684\\u7537\\u4eba\"},{\"open_url\":\"\\/group\\/6583049111459594760\\/\",\"group_id\":\"6583049111459594760\",\"image_url\":\"\\/\\/p1.pstatp.com\\/list\\/240x240\\/pgc-image\\/15327355613717121b05893\",\"title\":\"\\u8d38\\u6613\\u6469\\u64e6\\u4e4b\\u9645\\uff0c\\u4e00\\u573a\\u4e2d\\u7f8e\\u519b\\u4e8b\\u5408\\u4f5c\\u5373\\u5c06\\u767b\\u573a\"},{\"open_url\":\"\\/group\\/6583137428104544782\\/\",\"group_id\":\"6583137428104544782\",\"image_url\":\"\\/\\/p3.pstatp.com\\/list\\/240x240\\/pgc-image\\/15327559916423eac3e60e2\",\"title\":\"\\u4e2d\\u56fd\\u519b\\u673a\\u7a7f\\u8d8a\\u5bf9\\u9a6c\\u6d77\\u5ce1\\u540e\\u4e0d\\u4e45 \\u4e2d\\u56fd\\u6d77\\u519b\\u6218\\u8230\\u53c8\\u7d27\\u968f\\u5176\\u540e\\u6765\\u4e86\"},{\"open_url\":\"\\/group\\/6583189250651456003\\/\",\"group_id\":\"6583189250651456003\",\"image_url\":\"\\/\\/p1.pstatp.com\\/list\\/240x240\\/pgc-image\\/153276817044185b86bf571\",\"title\":\"\\u526f\\u7701\\u957f\\u517c\\u4efb\\u653f\\u5e9c\\u79d8\\u4e66\\u957f\\uff0c\\u4eca\\u5929\\u4efb\\u547d\"}]};\n";
//        Pattern pattern = Pattern.compile("var _data = \\{.*}");
//        Matcher matcher = pattern.matcher(str);
//        System.out.println(extractMessageByRegular("//www.sohu.com/a/245167821_461392"));
//        System.out.println(getUrl("            <li><a data-param=\"_f=index_chan08select_1\" title=\"美媒：火箭已非勇士最强敌 绿军或颠覆金州王朝\" href=\"//www.sohu.com/a/245154704_458722\" target=\"_blank\">美媒：火箭已非勇士最强敌 绿军或颠覆金州王朝</a></li>\n"));
    }
}
